<!DOCTYPE html>
<html>
<head><meta name="generator" content="Hexo 3.8.0">
  <meta charset="utf-8">
  

  
  <title>OpenrestyTCP反向代理-EQMX高可用 | </title>
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
  <meta name="baidu-site-verification" content="T7hOozd75A">
  <meta name="description" content="编译安装12345678tar -zxvf openresty-1.13.6.1.tar.gz./configuremakemake install Openresty配置按注释调整配置nginx.conf 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152">
<meta name="keywords" content="openresty,反向代理,MQTT,EMQX">
<meta property="og:type" content="article">
<meta property="og:title" content="OpenrestyTCP反向代理-EQMX高可用">
<meta property="og:url" content="http://lowezheng.gitee.io/2018/11/29/operesty/index.html">
<meta property="og:site_name">
<meta property="og:description" content="编译安装12345678tar -zxvf openresty-1.13.6.1.tar.gz./configuremakemake install Openresty配置按注释调整配置nginx.conf 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152">
<meta property="og:locale" content="zh-CN">
<meta property="og:updated_time" content="2018-12-04T00:49:31.000Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="OpenrestyTCP反向代理-EQMX高可用">
<meta name="twitter:description" content="编译安装12345678tar -zxvf openresty-1.13.6.1.tar.gz./configuremakemake install Openresty配置按注释调整配置nginx.conf 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152">
  
    <link rel="alternate" href="/atom.xml" title="" type="application/atom+xml">
  
  
    <link rel="icon" href="/favicon.png">
  
  
    <link href="//fonts.googleapis.com/css?family=Source+Code+Pro" rel="stylesheet" type="text/css">
  
  <link rel="stylesheet" href="/css/style.css">
</head>
</html>
<body>
  <div id="container">
    <div id="wrap">
      <header id="header">
  <div id="banner"></div>
  <div id="header-outer" class="outer">
    <div id="header-title" class="inner">
      <h1 id="logo-wrap">
        <a href="/" id="logo"></a>
      </h1>
      
    </div>
    <div id="header-inner" class="inner">
      <nav id="main-nav">
        <a id="main-nav-toggle" class="nav-icon"></a>
        
          <a class="main-nav-link" href="/">主页</a>
        
          <a class="main-nav-link" href="/archives">文档库</a>
        
      </nav>
      <nav id="sub-nav">
        
          <a id="nav-rss-link" class="nav-icon" href="/atom.xml" title="RSS Feed"></a>
        
        <a id="nav-search-btn" class="nav-icon" title="搜索"></a>
      </nav>
      <div id="search-form-wrap">
        <form target="_blank" action="http://zhannei.baidu.com/cse/site" class="search-form">
            <input type="text" name="q" class="search-form-input" size="30">
            <input type="hidden" name="cc" value="lowezheng.gitee.io">
        </form>
      </div>
    </div>
  </div>
</header>
      <div class="outer">
        <section id="main"><article id="post-operesty" class="article article-type-post" itemscope="" itemprop="blogPost">
  <div class="article-meta">
    <a href="/2018/11/29/operesty/" class="article-date">
  <time datetime="2018-11-29T09:02:12.000Z" itemprop="datePublished">2018-11-29</time>
</a>
    
  </div>
  <div class="article-inner">
    
    
      <header class="article-header">
        
  
    <h1 class="article-title" itemprop="name">
      OpenrestyTCP反向代理-EQMX高可用
    </h1>
  

      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <hr>
<h2 id="编译安装"><a href="#编译安装" class="headerlink" title="编译安装"></a>编译安装</h2><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line">tar -zxvf openresty-1.13.6.1.tar.gz</span><br><span class="line"></span><br><span class="line">./configure</span><br><span class="line"></span><br><span class="line">make</span><br><span class="line"></span><br><span class="line">make install</span><br></pre></td></tr></table></figure>
<h2 id="Openresty配置"><a href="#Openresty配置" class="headerlink" title="Openresty配置"></a>Openresty配置</h2><p><code>按注释调整配置</code><br>nginx.conf</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#user  nobody;</span></span><br><span class="line"><span class="comment">## 1.工作进程数，与CPU逻辑核心数保持一致(***必须配置***)</span></span><br><span class="line">worker_processes  8;</span><br><span class="line"><span class="comment">## 2. 提高TCP连接数(***必须配置***)</span></span><br><span class="line">worker_rlimit_nofile 1000000;</span><br><span class="line"></span><br><span class="line"><span class="comment">#error_log  logs/error.log;</span></span><br><span class="line"><span class="comment">#error_log  logs/error.log  notice;</span></span><br><span class="line"><span class="comment">#error_log  logs/error.log  info;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#pid        logs/nginx.pid;</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">events &#123;</span><br><span class="line">    <span class="comment">## 3.采用epoll事件模型(***必须配置***)</span></span><br><span class="line">    use epoll;</span><br><span class="line">    <span class="comment">## 4.提高TCP连接数(***必须配置***)</span></span><br><span class="line">    worker_connections 1000000;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">## 6.最好删除该部分监听端口，保留该部分为了查看nginx是否正常运行</span></span><br><span class="line">http &#123;</span><br><span class="line">    include       mime.types;</span><br><span class="line">    default_type  application/octet-stream;</span><br><span class="line"></span><br><span class="line">    <span class="comment">#log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '</span></span><br><span class="line">    <span class="comment">#                  '$status $body_bytes_sent "$http_referer" '</span></span><br><span class="line">    <span class="comment">#                  '"$http_user_agent" "$http_x_forwarded_for"';</span></span><br><span class="line"></span><br><span class="line">    <span class="comment">#access_log  logs/access.log  main;</span></span><br><span class="line"></span><br><span class="line">    sendfile        on;</span><br><span class="line">    <span class="comment">#tcp_nopush     on;</span></span><br><span class="line"></span><br><span class="line">    <span class="comment">#keepalive_timeout  0;</span></span><br><span class="line">    keepalive_timeout  65;</span><br><span class="line"></span><br><span class="line">    <span class="comment">#gzip  on;</span></span><br><span class="line"></span><br><span class="line">    server &#123;</span><br><span class="line">        listen       89;</span><br><span class="line">        server_name  localhost;</span><br><span class="line"></span><br><span class="line">        <span class="comment">#charset koi8-r;</span></span><br><span class="line"></span><br><span class="line">        <span class="comment">#access_log  logs/host.access.log  main;</span></span><br><span class="line"></span><br><span class="line">        location / &#123;</span><br><span class="line">            root   html;</span><br><span class="line">            index  index.html index.htm;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">## 4.导入EMQX的集群配置文件</span></span><br><span class="line">include stream.conf;</span><br></pre></td></tr></table></figure>
<p>stream.conf</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">stream &#123;</span><br><span class="line">    <span class="comment">## 1. 访问日志，排查问题使用</span></span><br><span class="line">	log_format basic <span class="string">'$remote_addr [$time_local] '</span></span><br><span class="line">					 <span class="string">'$protocol $status $bytes_sent $bytes_received '</span></span><br><span class="line">					 <span class="string">'$session_time $upstream_addr'</span>;</span><br><span class="line"></span><br><span class="line">	access_log logs/stream-access.log basic;</span><br><span class="line"></span><br><span class="line">    upstream emqx &#123;</span><br><span class="line">        <span class="comment">## 3.集群节点，可以配置多个，不用粘性会话，MQTT集群内是互通的，路由任意节#点都可以收到数据包</span></span><br><span class="line"></span><br><span class="line">        <span class="comment">#server 192.168.3.24:1883 max_fails=3 fail_timeout=30s;</span></span><br><span class="line">		server 192.168.3.23:1883 max_fails=3 fail_timeout=30s;</span><br><span class="line">    &#125;</span><br><span class="line">    server &#123;</span><br><span class="line">        <span class="comment">## 2.监听端口， so_keepalive必须配置，tcp保活时间控制2min,检测数据包间隔5s,重试3次，（MQTT的心跳包是1m发一个，保活时间间隔不得低于1m）</span></span><br><span class="line">        listen 90 so_keepalive=2m:2s:3;</span><br><span class="line">        proxy_connect_timeout 1s;</span><br><span class="line">        proxy_pass emqx;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h3 id="Linux系统参数调整"><a href="#Linux系统参数调整" class="headerlink" title="Linux系统参数调整"></a>Linux系统参数调整</h3><p>需要设置nginx–&gt;EMQX的端口连接上限，理论上nginx和EQM集群节点最多保持65535个连接,排除已被占用端口，预估可达到6W。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">sysctl -w net.ipv4.ip_local_port_range=<span class="string">"1024 65535"</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 所有进程最大打开文件数(2M)</span></span><br><span class="line"></span><br><span class="line">sysctl -w fs.file-max=2097152</span><br><span class="line"></span><br><span class="line"><span class="comment"># 单进程最大可分配文件数(2M)</span></span><br><span class="line"></span><br><span class="line">sysctl -w fs.nr_open=2097152</span><br></pre></td></tr></table></figure>
<p>在/etc/security/limits.conf 添加，<code>关闭当前会话，重新打开，否则不生效</code></p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">* soft nofile 100000</span><br><span class="line"></span><br><span class="line">* hard nofile 100000</span><br></pre></td></tr></table></figure>
<p>端口的TCP监听队列长度，可能用于处理大量TCP连接时，队列溢出</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">sysctl -w net.core.somaxconn=65535</span><br></pre></td></tr></table></figure>
<h3 id="压测准备"><a href="#压测准备" class="headerlink" title="压测准备"></a>压测准备</h3><p>Client(1台)—&gt;Nginx(1台)—&gt;EMQX(1台)网络环境，理论最大支持6W客户端在线</p>
<p>如果希望达到百万级别的环境:</p>
<p>准备Client(16台)—&gt;Nginx(16台)—-&gt;EMQX(1台)</p>
<p>或者Client(16台)—&gt;EMQX(1台)</p>

      
    </div>
    <footer class="article-footer">
      <a data-url="http://lowezheng.gitee.io/2018/11/29/operesty/" data-id="cjp9hvybf00055yf04780iz6v" class="article-share-link">Share</a>
      
      
  <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/EMQX/">EMQX</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/MQTT/">MQTT</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/openresty/">openresty</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/反向代理/">反向代理</a></li></ul>

    </footer>
  </div>
  
    
<nav id="article-nav">
  
    <a href="/2018/12/01/zookeeper/" id="article-nav-newer" class="article-nav-link-wrap">
      <strong class="article-nav-caption">Newer</strong>
      <div class="article-nav-title">
        
          Zookeeper安装
        
      </div>
    </a>
  
  
    <a href="/2018/11/29/openssl/" id="article-nav-older" class="article-nav-link-wrap">
      <strong class="article-nav-caption">Older</strong>
      <div class="article-nav-title">Openssl自建CA</div>
    </a>
  
</nav>

  
</article>

</section>
        
          <aside id="sidebar">
  
    

  
    
  <div class="widget-wrap">
    <h3 class="widget-title">标签</h3>
    <div class="widget">
      <ul class="tag-list"><li class="tag-list-item"><a class="tag-list-link" href="/tags/EMQX/">EMQX</a></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/MQTT/">MQTT</a></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/ca/">ca</a></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/openresty/">openresty</a></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/openssl/">openssl</a></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/tomcat/">tomcat</a></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/zookeeper/">zookeeper</a></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/内存溢出/">内存溢出</a></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/反向代理/">反向代理</a></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/安装/">安装</a></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/自建证书/">自建证书</a></li></ul>
    </div>
  </div>


  
    
  <div class="widget-wrap">
    <h3 class="widget-title">标签云</h3>
    <div class="widget tagcloud">
      <a href="/tags/EMQX/" style="font-size: 10px;">EMQX</a> <a href="/tags/MQTT/" style="font-size: 10px;">MQTT</a> <a href="/tags/ca/" style="font-size: 10px;">ca</a> <a href="/tags/openresty/" style="font-size: 10px;">openresty</a> <a href="/tags/openssl/" style="font-size: 10px;">openssl</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/zookeeper/" style="font-size: 10px;">zookeeper</a> <a href="/tags/内存溢出/" style="font-size: 10px;">内存溢出</a> <a href="/tags/反向代理/" style="font-size: 10px;">反向代理</a> <a href="/tags/安装/" style="font-size: 10px;">安装</a> <a href="/tags/自建证书/" style="font-size: 10px;">自建证书</a>
    </div>
  </div>

  
    
  <div class="widget-wrap">
    <h3 class="widget-title">归档</h3>
    <div class="widget">
      <ul class="archive-list"><li class="archive-list-item"><a class="archive-list-link" href="/archives/2018/12/">十二月 2018</a></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2018/11/">十一月 2018</a></li></ul>
    </div>
  </div>


  
    
  <div class="widget-wrap">
    <h3 class="widget-title">最新文章</h3>
    <div class="widget">
      <ul>
        
          <li>
            <a href="/2018/12/04/mac-command/">Mac指令集合</a>
          </li>
        
          <li>
            <a href="/2018/12/01/zookeeper/">Zookeeper安装</a>
          </li>
        
          <li>
            <a href="/2018/11/29/operesty/">OpenrestyTCP反向代理-EQMX高可用</a>
          </li>
        
          <li>
            <a href="/2018/11/29/openssl/">Openssl自建CA</a>
          </li>
        
          <li>
            <a href="/2018/11/29/tomcat/">Tomcat内存溢出跟踪</a>
          </li>
        
      </ul>
    </div>
  </div>

  
</aside>
        
      </div>
      <footer id="footer">
  
  <div class="outer">
    <div id="footer-info" class="inner">
      &copy; 2018 lowe zheng<br>
      Powered by <a href="http://hexo.io/" target="_blank">Hexo</a>
    </div>
  </div>
</footer>
    </div>
    <nav id="mobile-nav">
  
    <a href="/" class="mobile-nav-link">主页</a>
  
    <a href="/archives" class="mobile-nav-link">文档库</a>
  
</nav>
    

<script src="https://code.jquery.com/jquery-2.2.4.min.js" integrity="sha256-BbhdlvQf/xTY9gja0Dq3HiwQF8LaCRTXxZKRutelT44=" crossorigin="anonymous"></script>


  <link rel="stylesheet" href="/fancybox/jquery.fancybox.css">
  <script src="/fancybox/jquery.fancybox.pack.js"></script>


<script src="/js/script.js"></script>



  </div>
</body>
</html>